package engine

import (
	"crowler/fetcher"
	"log"
	"database/sql"
)


var (
	db *sql.DB
)

func init() {
	var err error
	db, err = sql.Open("mysql", "root:root@tcp(127.0.0.1:3306)/xiangqin?charset=utf8")
	if err != nil {
		log.Println(err)
	}
}

func Run(seeds ...Request)  {

	var requests []Request

	for _,r := range seeds {
        requests  = append(requests,r)
	}

	for len(requests) > 0{
		r := requests[0]
		requests = requests[1:]

		log.Printf("Fetching %s", r.Url)

		body,err := fetcher.Fetch(r.Url)

		if err != nil{
			log.Printf("fetcher:error" + "fetching url %s : %v" , r.Url , err)
			continue
		}

		parseResult := r.ParsetFunc(body)
		requests = append(requests , parseResult.Requests...) //把收集到的request 加入队列中

		for _,item := range parseResult.Items {
			log.Printf("Got item %v " , item)
		}



	}
}